# -*- coding: utf-8 -*-
"""
Created on Sat Oct 20 11:35:23 2018

@author: HP
"""
import numpy as np
import pandas as pd
from my_data_describe import my_data_describe
import matplotlib.pyplot as plt
from my_boxplot import my_boxplot, my_boxplot_imbalance
from sklearn.linear_model import LogisticRegression
# 读文件
train_file = pd.read_csv('../input/train.csv',index_col=0)
test_file = pd.read_csv('../input/test.csv',index_col=0)
train_X = train_file.iloc[:,:-1]
train_y =  train_file.iloc[:,-1]
test_X = test_file

# 对dataframe进行增删改时，在这个上面进行
ntrain = train_X.shape[0]
train_test = pd.concat((train_X, test_X)).reset_index(drop=True)

#%% 处理缺失数据
### 缺失过多的填固定值或删去
df_describe = my_data_describe(train_test)
na_lot = df_describe[df_describe.na_rate > 0.4].index
my_boxplot_imbalance(train_X[na_lot],train_y, 0.4)
# 根据上图选出填充列和待删列
'''
fillna: 'FireplaceQu','Fence','Alley'
drop:   'MiscFeature','PoolQC'
'''
fill_None_li = ['FireplaceQu','Fence','Alley']
drop_li = ['MiscFeature','PoolQC']
# 试了下效果不理想 先直接删除这几列吧
#train_test[fill_None_li] = train_test[fill_None_li].fillna('None')
train_test = train_test.drop(drop_li,axis=1)
train_test = train_test.drop(fill_None_li,axis=1)
### 缺失较少的 模型预测填充
# 类别型预测
from my_fillna_model import my_fillna_model
train_test = my_fillna_model(train_test)
df_describe = my_data_describe(train_test)

# MSSubClass

#%% 处理极偏数据
imbalance_li = my_boxplot_imbalance(train_test[:ntrain], np.log(train_y))
drop_skli = ['Utilities','MiscVal']
'''
1   0
=0 !=0    LowQualFinSF PoolArea
类别型     Heating RoofMatl Condition2 Street GarageCond GarageQual
'''
train_test = train_test.drop(drop_skli,axis=1)
for i in ['LowQualFinSF', 'PoolArea']:
    s = train_test[i]
    s[s != 0] = 'a'
    s[s == 0] = 'b'
    s[s == 'a'] = 0
    s[s == 'b'] = 1
# 类别(类型全是object)
class_li = ['Heating', 'RoofMatl', 'Condition2', 'Street', 'GarageCond', 'GarageQual']

#%%
# 数值类型转类别型 .astype('str')
### 处理数值型
df_describe = my_data_describe(train_test)
class_num_li = df_describe[(df_describe.unique_num<20)&(df_describe.type!='object')].index
#画出箱型图
for i in class_num_li:
    x=train_test[:ntrain][i]
    y=train_y
    my_boxplot(x,y,i)
'''
drop_linum = ['YrSold','KitchenAbvGr']
as_str_li = ['MoSold', 'MSSubClass']
不改变 GarageCars  TotRmsAbvGrd BedroomAbvGrd
'''
drop_linum = ['YrSold','KitchenAbvGr']
as_str_li = ['MoSold', 'MSSubClass']
train_test = train_test.drop(drop_linum,axis=1)
for i in as_str_li:
    train_test[i] = train_test[i].astype(str)
### 处理类别型
df_describe = my_data_describe(train_test)
class_obj_li = df_describe[df_describe.type=='object'].index
train_test = pd.get_dummies(train_test)

#%%
from scipy.special import boxcox1p
sk = train_test.skew()
skewness = sk[sk.abs()>0.75].index
skewed_features = skewness
lam = 0.15
for feat in skewed_features:
    #all_data[feat] += 1
    train_test[feat] = boxcox1p(train_test[feat], lam)
drop_li = sk[sk.abs()>45].index
train_test_all = train_test.drop(drop_li,axis=1)

train_test_all.columns
my_data_describe(train_test_all).na_rate


train_X = train_test_all[:ntrain]
train_y = train_y.reset_index(drop=True)
pd.concat([train_X,train_y],axis=1).to_csv('./input_processed/train.csv')
train_test_all[ntrain:].to_csv('./input_processed/test.csv')

